# Updated by Wei on 2014/06/13 at school
# attempt to fit for the clueweb09B dataset
# Updated by Wei on 2014/01/08 at school
from __future__ import division
import random
import math
import os
import sys
from sets import Set

# make the file term Piece Info For whole clueweb09B lexicon with stepGap to be 2
# key: list length class label
# value: # of ranges
listLengthClassWithNumOfRangesDict = {}
# key: list length class label
# value: lower bound threshold for this class
listLengthClassWithLowerBoundThresholdDict = {}
inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/data/secondFactorProbability/predefinedClassLabelLowerBoundOfListLength_clueweb09B_20140612"
inputFileHandler = open(inputFileName0,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentListLengthClassLabel = int(lineElements[0])
    currentLowerBoundThreshold = int(lineElements[1])
    currentNumOfRanges = int(lineElements[2])
    listLengthClassWithLowerBoundThresholdDict[currentListLengthClassLabel] = currentLowerBoundThreshold
    listLengthClassWithNumOfRangesDict[currentListLengthClassLabel] = currentNumOfRanges 
inputFileHandler.close()
# print "len(listLengthClassWithLowerBoundThresholdDict):",len(listLengthClassWithLowerBoundThresholdDict)
# print "len(listLengthClassWithNumOfRangesDict):",len(listLengthClassWithNumOfRangesDict)

inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09BIndexOverallStatistics_sortedByListLength_20140606"
inputFileHandler = open(inputFileName1,"r")
currentLine = inputFileHandler.readline()
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    currentListLength = int(currentLineElements[2])
    outputLine = ""
    for i in range(0,76):
        if currentListLength >= listLengthClassWithLowerBoundThresholdDict[i] and currentListLength < listLengthClassWithLowerBoundThresholdDict[i+1]:
            currentNumOfRanges = listLengthClassWithNumOfRangesDict[i]
            break
    outputLine = str(currentTermID) + " " + str(currentTerm) + " " + str(currentListLength) + " " + str(i) + " " + str(currentNumOfRanges) + " "
    numOfPostingsInCurrentRange = 0
    numOfPostingsLeft = currentListLength
    for i in range(0,currentNumOfRanges-1):
        numOfPostingsInCurrentRange = int(numOfPostingsLeft / 2)
        numOfPostingsLeft = numOfPostingsLeft - numOfPostingsInCurrentRange
        outputLine += str(i) + " " + str(numOfPostingsInCurrentRange) + " "
    outputLine += str(currentNumOfRanges-1) + " " + str(numOfPostingsLeft)
    outputLine = outputLine.strip()
    print outputLine
    currentLine = inputFileHandler.readline()
inputFileHandler.close()
exit(1)
